import pandas as pd
# plotting modules
from matplotlib import pyplot as plt
import plotly.express as px
import plotly.offline as po
import seaborn as sns
import numpy as np
po.offline.init_notebook_mode()
fmnist_df_train = pd.read_csv("data/fashion_mnist/fashion-mnist_train.csv")
fmnist_df_test = pd.read_csv("data/fashion_mnist/fashion-mnist_test.csv")
y_train = fmnist_df_train["label"].to_numpy()
X_train = fmnist_df_train.drop("label", axis=1).to_numpy()
y_test = fmnist_df_test["label"].to_numpy()
X_test = fmnist_df_test.drop("label", axis=1).to_numpy()
print(f'Image DType: {type(X_train)}')
print(f'Image Element DType: {type(X_train[0,0])}')
print(f'Label Element DType: {type(y_train[0])}')
print('**Shapes:**')
print('Train Data:')
print(f'Images: {X_train.shape}')
print(f'Labels: {y_train.shape}')
print('Test Data:') # the text images should be a random sample of the overall test set, and hence should have the same type, shape and image-size as the overall train set
print(f'Images: {X_test.shape}')
print(f'Labels: {y_test.shape}')
print('Image Data Range:')
print(f'Min: {X_train.min()}')
print(f'Max: {X_train.max()}')
print('Class Label Range:')
print(f'Min: {y_train.min()}')
print(f'Max: {y_train.max()}')
Image DType: <class 'numpy.ndarray'> Image Element DType: <class 'numpy.int64'> Label Element DType: <class 'numpy.int64'> **Shapes:** Train Data: Images: (60000, 784) Labels: (60000,) Test Data: Images: (10000, 784) Labels: (10000,) Image Data Range: Min: 0 Max: 255 Class Label Range: Min: 0 Max: 9
print(X_train[0])
print(y_train[:10])
[ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 62 61 21 29 23 51 136 61 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 88 201 228 225 255 115 62 137 255 235 222 255 135 0 0 0 0 0 0 0 0 0 0 0 0 0 47 252 234 238 224 215 215 229 108 180 207 214 224 231 249 254 45 0 0 0 0 0 0 0 0 1 0 0 214 222 210 213 224 225 217 220 254 233 219 221 217 223 221 240 254 0 0 1 0 0 0 1 0 0 0 128 237 207 224 224 207 216 214 210 208 211 221 208 219 213 226 211 237 150 0 0 0 0 0 0 2 0 0 237 222 215 207 210 212 213 206 214 213 214 213 210 215 214 206 199 218 255 13 0 2 0 0 0 4 0 85 228 210 218 200 211 208 203 215 210 209 209 210 213 211 210 217 206 213 231 175 0 0 0 0 0 0 0 217 224 215 206 205 204 217 230 222 215 224 233 228 232 228 224 207 212 215 213 229 31 0 4 0 1 0 21 225 212 212 203 211 225 193 139 136 195 147 156 139 128 162 197 223 207 220 213 232 177 0 0 0 0 0 123 226 207 211 209 205 228 158 90 103 186 138 100 121 147 158 183 226 208 214 209 216 255 13 0 1 0 0 226 219 202 208 206 205 216 184 156 150 193 170 164 168 188 186 200 219 216 213 213 211 233 148 0 0 0 45 227 204 214 211 218 222 221 230 229 221 213 224 233 226 220 219 221 224 223 217 210 218 213 254 0 0 0 157 226 203 207 211 209 215 205 198 207 208 201 201 197 203 205 210 207 213 214 214 214 213 208 234 107 0 0 235 213 204 211 210 209 213 202 197 204 215 217 213 212 210 206 212 203 211 218 215 214 208 209 222 230 0 52 255 207 200 208 213 210 210 208 207 202 201 209 216 216 216 216 214 212 205 215 201 228 208 214 212 218 25 118 217 201 206 208 213 208 205 206 210 211 202 199 207 208 209 210 207 210 210 245 139 119 255 202 203 236 114 171 238 212 203 220 216 217 209 207 205 210 211 206 204 206 209 211 215 210 206 221 242 0 224 234 230 181 26 39 145 201 255 157 115 250 200 207 206 207 213 216 206 205 206 207 206 215 207 221 238 0 0 188 85 0 0 0 0 0 31 0 129 253 190 207 208 208 208 209 211 211 209 209 209 212 201 226 165 0 0 0 0 0 0 2 0 0 0 0 89 254 199 199 192 196 198 199 201 202 203 204 203 203 200 222 155 0 3 3 3 2 0 0 0 1 5 0 0 255 218 226 232 228 224 222 220 219 219 217 221 220 212 236 95 0 2 0 0 0 0 0 0 0 0 0 0 155 194 168 170 171 173 173 179 177 175 172 171 167 161 180 0 0 1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] [2 9 6 0 3 4 4 5 4 8]
def plot_image(image_data):
image = image_data.reshape(28, 28)
plt.imshow(image, cmap="binary")
plt.axis("off")
plt.figure(figsize=(9, 9))
for idx, image_data in enumerate(X_train[:100]):
plt.subplot(10, 10, idx + 1)
plot_image(image_data)
plt.subplots_adjust(wspace=0, hspace=0)
plt.show()
y_train[:100]
array([2, 9, 6, 0, 3, 4, 4, 5, 4, 8, 0, 8, 9, 0, 2, 2, 9, 3, 3, 3, 8, 7,
4, 4, 0, 4, 4, 8, 7, 1, 5, 0, 5, 3, 2, 7, 3, 4, 2, 1, 6, 0, 9, 6,
0, 5, 6, 7, 7, 2, 5, 2, 2, 4, 1, 4, 9, 8, 3, 4, 5, 5, 6, 3, 5, 8,
5, 9, 8, 1, 2, 8, 1, 3, 6, 8, 3, 4, 2, 5, 0, 2, 6, 8, 1, 2, 7, 6,
6, 4, 6, 5, 0, 1, 7, 3, 5, 8, 4, 3], dtype=int64)
To verify the images, they were displayed at different scales.
fig = plt.figure()
plt.subplot(1, 2, 1)
plt.imshow(X_train[0].reshape(28, 28), cmap='gray')
plt.title('Gray Scale')
plt.subplot(1, 2, 2)
plt.imshow(X_train[0].reshape(28, 28), cmap='binary')
plt.title('Binary Scale')
Text(0.5, 1.0, 'Binary Scale')
To inspect the color based on each pixel value, the data are visualized.
def visualize_input(i, ax, y):
img = X_train[i].reshape(28,28)
ax.imshow(img, cmap='gray')
ax.set_title(f"Class no : {y[i]}")
width, height = img.shape
thresh = img.max()/2.5
for x in range(width):
for y in range(height):
ax.annotate(str(img[x][y]), xy=(y,x),
horizontalalignment='center',
verticalalignment='center',
color='white' if img[x][y]<thresh else 'black')
ax.set_xticks([])
ax.set_yticks([])
for i in range(2):
fig = plt.figure(figsize = (12,12))
ax = fig.add_subplot(111)
visualize_input(i, ax, y_train)
# Plot the distribution of pixel values
fig, axes = plt.subplots(1, 5, figsize=(20, 5))
for i in range(5):
plt.sca(axes[i])
plt.hist(X_train[i], bins=50, edgecolor='black')
plt.title(f'Pixel Value Distribution for Class {y_train[i]}')
plt.xlabel('Pixel Value')
plt.ylabel('Count')
plt.tight_layout()
plt.show()
fig, axes = plt.subplots(1, 10, figsize=(20, 2))
for digit in range(10):
digit_indices = np.where(y_train.astype('int8') == digit)[0]
avg_image = np.mean(X_train[digit_indices], axis=0).reshape(28, 28)
axes[digit].imshow(avg_image, cmap='gray')
axes[digit].set_title(str(digit))
axes[digit].axis('off')
plt.show()
y_values, counts = np.unique(y_train, return_counts=True)
df_y = pd.DataFrame({'class': y_values, 'count': counts})
sns.barplot(df_y, x="class", y="count")
<Axes: xlabel='class', ylabel='count'>
# Count the occurrences of each class
class_counts = np.bincount(y_train)
# Plot a piechart using plotly
fig = px.pie(values=class_counts, names=[str(i) for i in range(10)], title='Percentage of samples per label')
fig.show()
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from keras.callbacks import ModelCheckpoint
Pixel values range up to 255, but to facilitate efficient training, the data is normalized by dividing by 255.
X_train = X_train.astype("float32") / 255.0
X_test = X_test.astype("float32") / 255.0
print(X_train.min())
print(X_train.max())
print(X_train.shape)
print(X_train[0].shape)
0.0 1.0 (60000, 784) (784,)
from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.2, random_state=42)
model_1 = keras.Sequential([
layers.Dense(512, activation="relu"),
layers.Dense(10, activation="softmax")
])
model_2 = keras.Sequential([
layers.Dense(512, activation="relu"),
layers.Dense(10, activation="softmax")
])
model_3 = keras.Sequential([
layers.Dense(512, activation="relu"),
layers.Dense(128, activation="relu"),
layers.Dense(10, activation="softmax")
])
model_1.compile(optimizer="rmsprop",
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
model_2.compile(optimizer="rmsprop",
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
model_3.compile(optimizer="rmsprop",
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
model_checkpoint_callback_1 = ModelCheckpoint(
filepath='data/best_model_1.hdf5',
save_weights_only=False, # Optionally save only weights instead of the full model
monitor='val_loss',
mode='min', # Monitor validation loss and save on minimum
save_best_only=True)
model_checkpoint_callback_2 = ModelCheckpoint(
filepath='data/best_model_2.hdf5',
save_weights_only=False, # Optionally save only weights instead of the full model
monitor='val_loss',
mode='min', # Monitor validation loss and save on minimum
save_best_only=True)
model_checkpoint_callback_3 = ModelCheckpoint(
filepath='data/best_model_3.hdf5',
save_weights_only=False, # Optionally save only weights instead of the full model
monitor='val_loss',
mode='min', # Monitor validation loss and save on minimum
save_best_only=True)
history_1 = model_1.fit(X_train,
y_train,
epochs=30,
batch_size=128,
validation_data=(X_val, y_val),
callbacks=[model_checkpoint_callback_1])
Epoch 1/30 375/375 [==============================] - 2s 6ms/step - loss: 0.5917 - accuracy: 0.7916 - val_loss: 0.4267 - val_accuracy: 0.8486 Epoch 2/30 375/375 [==============================] - 2s 6ms/step - loss: 0.4103 - accuracy: 0.8498 - val_loss: 0.3892 - val_accuracy: 0.8543 Epoch 3/30 375/375 [==============================] - 2s 6ms/step - loss: 0.3588 - accuracy: 0.8693 - val_loss: 0.3544 - val_accuracy: 0.8699 Epoch 4/30 375/375 [==============================] - 2s 6ms/step - loss: 0.3277 - accuracy: 0.8806 - val_loss: 0.3496 - val_accuracy: 0.8698 Epoch 5/30 375/375 [==============================] - 2s 6ms/step - loss: 0.3091 - accuracy: 0.8864 - val_loss: 0.3255 - val_accuracy: 0.8800 Epoch 6/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2906 - accuracy: 0.8931 - val_loss: 0.3473 - val_accuracy: 0.8740 Epoch 7/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2765 - accuracy: 0.8975 - val_loss: 0.3329 - val_accuracy: 0.8702 Epoch 8/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2651 - accuracy: 0.9032 - val_loss: 0.3222 - val_accuracy: 0.8813 Epoch 9/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2541 - accuracy: 0.9051 - val_loss: 0.3197 - val_accuracy: 0.8849 Epoch 10/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2423 - accuracy: 0.9105 - val_loss: 0.3283 - val_accuracy: 0.8832 Epoch 11/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2329 - accuracy: 0.9138 - val_loss: 0.3272 - val_accuracy: 0.8878 Epoch 12/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2248 - accuracy: 0.9163 - val_loss: 0.3360 - val_accuracy: 0.8900 Epoch 13/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2176 - accuracy: 0.9191 - val_loss: 0.3169 - val_accuracy: 0.8894 Epoch 14/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2096 - accuracy: 0.9208 - val_loss: 0.3686 - val_accuracy: 0.8828 Epoch 15/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2031 - accuracy: 0.9247 - val_loss: 0.3310 - val_accuracy: 0.8896 Epoch 16/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1976 - accuracy: 0.9268 - val_loss: 0.3227 - val_accuracy: 0.8913 Epoch 17/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1905 - accuracy: 0.9288 - val_loss: 0.3287 - val_accuracy: 0.8917 Epoch 18/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1846 - accuracy: 0.9330 - val_loss: 0.3220 - val_accuracy: 0.8927 Epoch 19/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1784 - accuracy: 0.9342 - val_loss: 0.3476 - val_accuracy: 0.8918 Epoch 20/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1742 - accuracy: 0.9349 - val_loss: 0.3355 - val_accuracy: 0.8940 Epoch 21/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1689 - accuracy: 0.9377 - val_loss: 0.3742 - val_accuracy: 0.8809 Epoch 22/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1624 - accuracy: 0.9386 - val_loss: 0.3826 - val_accuracy: 0.8802 Epoch 23/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1596 - accuracy: 0.9411 - val_loss: 0.4131 - val_accuracy: 0.8800 Epoch 24/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1558 - accuracy: 0.9431 - val_loss: 0.3493 - val_accuracy: 0.8886 Epoch 25/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1510 - accuracy: 0.9444 - val_loss: 0.4120 - val_accuracy: 0.8861 Epoch 26/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1465 - accuracy: 0.9463 - val_loss: 0.3669 - val_accuracy: 0.8892 Epoch 27/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1426 - accuracy: 0.9486 - val_loss: 0.3960 - val_accuracy: 0.8865 Epoch 28/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1386 - accuracy: 0.9497 - val_loss: 0.3918 - val_accuracy: 0.8900 Epoch 29/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1365 - accuracy: 0.9509 - val_loss: 0.4002 - val_accuracy: 0.8895 Epoch 30/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1330 - accuracy: 0.9523 - val_loss: 0.3964 - val_accuracy: 0.8914
history_2 = model_2.fit(X_train,
y_train,
epochs=30,
batch_size=32,
validation_data=(X_val, y_val),
callbacks=[model_checkpoint_callback_2])
Epoch 1/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.5175 - accuracy: 0.8136 - val_loss: 0.4172 - val_accuracy: 0.8505 Epoch 2/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.3860 - accuracy: 0.8613 - val_loss: 0.3952 - val_accuracy: 0.8570 Epoch 3/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.3507 - accuracy: 0.8751 - val_loss: 0.3599 - val_accuracy: 0.8712 Epoch 4/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.3308 - accuracy: 0.8827 - val_loss: 0.3581 - val_accuracy: 0.8788 Epoch 5/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.3179 - accuracy: 0.8893 - val_loss: 0.3856 - val_accuracy: 0.8692 Epoch 6/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.3021 - accuracy: 0.8939 - val_loss: 0.3761 - val_accuracy: 0.8702 Epoch 7/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2954 - accuracy: 0.8978 - val_loss: 0.3806 - val_accuracy: 0.8783 Epoch 8/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2847 - accuracy: 0.8997 - val_loss: 0.3741 - val_accuracy: 0.8763 Epoch 9/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2782 - accuracy: 0.9024 - val_loss: 0.3682 - val_accuracy: 0.8907 Epoch 10/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2713 - accuracy: 0.9074 - val_loss: 0.3984 - val_accuracy: 0.8838 Epoch 11/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2647 - accuracy: 0.9111 - val_loss: 0.3949 - val_accuracy: 0.8871 Epoch 12/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2591 - accuracy: 0.9144 - val_loss: 0.4014 - val_accuracy: 0.8826 Epoch 13/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2533 - accuracy: 0.9150 - val_loss: 0.3855 - val_accuracy: 0.8896 Epoch 14/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2487 - accuracy: 0.9166 - val_loss: 0.3880 - val_accuracy: 0.8860 Epoch 15/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2451 - accuracy: 0.9196 - val_loss: 0.4164 - val_accuracy: 0.8846 Epoch 16/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2415 - accuracy: 0.9197 - val_loss: 0.4389 - val_accuracy: 0.8884 Epoch 17/30 1500/1500 [==============================] - 7s 4ms/step - loss: 0.2390 - accuracy: 0.9210 - val_loss: 0.4832 - val_accuracy: 0.8832 Epoch 18/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2359 - accuracy: 0.9240 - val_loss: 0.5083 - val_accuracy: 0.8661 Epoch 19/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2321 - accuracy: 0.9258 - val_loss: 0.4709 - val_accuracy: 0.8864 Epoch 20/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2280 - accuracy: 0.9269 - val_loss: 0.4750 - val_accuracy: 0.8834 Epoch 21/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2249 - accuracy: 0.9295 - val_loss: 0.5169 - val_accuracy: 0.8814 Epoch 22/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2201 - accuracy: 0.9321 - val_loss: 0.4800 - val_accuracy: 0.8856 Epoch 23/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2174 - accuracy: 0.9327 - val_loss: 0.6002 - val_accuracy: 0.8844 Epoch 24/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2141 - accuracy: 0.9333 - val_loss: 0.5453 - val_accuracy: 0.8769 Epoch 25/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2092 - accuracy: 0.9352 - val_loss: 0.5319 - val_accuracy: 0.8799 Epoch 26/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2086 - accuracy: 0.9368 - val_loss: 0.5243 - val_accuracy: 0.8770 Epoch 27/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2065 - accuracy: 0.9363 - val_loss: 0.5347 - val_accuracy: 0.8903 Epoch 28/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2032 - accuracy: 0.9387 - val_loss: 0.5857 - val_accuracy: 0.8884 Epoch 29/30 1500/1500 [==============================] - 6s 4ms/step - loss: 0.2070 - accuracy: 0.9395 - val_loss: 0.5658 - val_accuracy: 0.8897 Epoch 30/30 1500/1500 [==============================] - 7s 4ms/step - loss: 0.2018 - accuracy: 0.9399 - val_loss: 0.5311 - val_accuracy: 0.8913
history_3 = model_3.fit(X_train,
y_train,
epochs=30,
batch_size=128,
validation_data=(X_val, y_val),
callbacks=[model_checkpoint_callback_3])
Epoch 1/30 375/375 [==============================] - 3s 7ms/step - loss: 0.5783 - accuracy: 0.7926 - val_loss: 0.4709 - val_accuracy: 0.8218 Epoch 2/30 375/375 [==============================] - 2s 6ms/step - loss: 0.3982 - accuracy: 0.8540 - val_loss: 0.4217 - val_accuracy: 0.8420 Epoch 3/30 375/375 [==============================] - 2s 6ms/step - loss: 0.3518 - accuracy: 0.8685 - val_loss: 0.3746 - val_accuracy: 0.8633 Epoch 4/30 375/375 [==============================] - 2s 6ms/step - loss: 0.3245 - accuracy: 0.8785 - val_loss: 0.3487 - val_accuracy: 0.8741 Epoch 5/30 375/375 [==============================] - 2s 6ms/step - loss: 0.3013 - accuracy: 0.8885 - val_loss: 0.3351 - val_accuracy: 0.8794 Epoch 6/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2830 - accuracy: 0.8937 - val_loss: 0.3485 - val_accuracy: 0.8698 Epoch 7/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2702 - accuracy: 0.8993 - val_loss: 0.3238 - val_accuracy: 0.8849 Epoch 8/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2590 - accuracy: 0.9029 - val_loss: 0.3187 - val_accuracy: 0.8882 Epoch 9/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2489 - accuracy: 0.9080 - val_loss: 0.2983 - val_accuracy: 0.8908 Epoch 10/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2375 - accuracy: 0.9088 - val_loss: 0.3332 - val_accuracy: 0.8844 Epoch 11/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2281 - accuracy: 0.9146 - val_loss: 0.3331 - val_accuracy: 0.8868 Epoch 12/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2225 - accuracy: 0.9149 - val_loss: 0.3568 - val_accuracy: 0.8842 Epoch 13/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2114 - accuracy: 0.9195 - val_loss: 0.3532 - val_accuracy: 0.8877 Epoch 14/30 375/375 [==============================] - 2s 6ms/step - loss: 0.2059 - accuracy: 0.9225 - val_loss: 0.3662 - val_accuracy: 0.8863 Epoch 15/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1984 - accuracy: 0.9243 - val_loss: 0.3521 - val_accuracy: 0.8841 Epoch 16/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1932 - accuracy: 0.9266 - val_loss: 0.4089 - val_accuracy: 0.8888 Epoch 17/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1871 - accuracy: 0.9284 - val_loss: 0.3845 - val_accuracy: 0.8885 Epoch 18/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1839 - accuracy: 0.9309 - val_loss: 0.4102 - val_accuracy: 0.8871 Epoch 19/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1758 - accuracy: 0.9335 - val_loss: 0.4247 - val_accuracy: 0.8892 Epoch 20/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1740 - accuracy: 0.9341 - val_loss: 0.4310 - val_accuracy: 0.8847 Epoch 21/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1679 - accuracy: 0.9370 - val_loss: 0.4406 - val_accuracy: 0.8729 Epoch 22/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1654 - accuracy: 0.9367 - val_loss: 0.4321 - val_accuracy: 0.8927 Epoch 23/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1598 - accuracy: 0.9390 - val_loss: 0.4484 - val_accuracy: 0.8852 Epoch 24/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1537 - accuracy: 0.9427 - val_loss: 0.4423 - val_accuracy: 0.8919 Epoch 25/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1523 - accuracy: 0.9421 - val_loss: 0.4630 - val_accuracy: 0.8917 Epoch 26/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1501 - accuracy: 0.9444 - val_loss: 0.4321 - val_accuracy: 0.8963 Epoch 27/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1479 - accuracy: 0.9452 - val_loss: 0.4662 - val_accuracy: 0.8856 Epoch 28/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1468 - accuracy: 0.9437 - val_loss: 0.4526 - val_accuracy: 0.8907 Epoch 29/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1437 - accuracy: 0.9479 - val_loss: 0.4373 - val_accuracy: 0.8957 Epoch 30/30 375/375 [==============================] - 2s 6ms/step - loss: 0.1398 - accuracy: 0.9485 - val_loss: 0.4893 - val_accuracy: 0.8897
history_dict_1 = history_1.history
# plot the training and validation loss
import matplotlib.pyplot as plt
plt.figure(figsize=(9, 5))
loss_values = history_dict_1["loss"]
val_loss_values = history_dict_1["val_loss"]
epochs = range(1, len(loss_values) + 1)
plt.plot(epochs, loss_values, "bo", label="Training loss")
plt.plot(epochs, val_loss_values, "ro", label="Validation loss")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Loss")
plt.legend()
plt.title("Training and validation loss")
Text(0.5, 1.0, 'Training and validation loss')
# plot the training and validation accuracy
import matplotlib.pyplot as plt
plt.figure(figsize=(9, 5))
loss_values = history_dict_1["accuracy"]
val_loss_values = history_dict_1["val_accuracy"]
epochs = range(1, len(loss_values) + 1)
plt.plot(epochs, loss_values, "bo", label="Training accuracy")
plt.plot(epochs, val_loss_values, "ro", label="Validation accuracy")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Accuracy")
plt.legend()
plt.title("Training and validation accuracy")
Text(0.5, 1.0, 'Training and validation accuracy')
history_dict_2 = history_2.history
# plot the training and validation loss
import matplotlib.pyplot as plt
plt.figure(figsize=(9, 5))
loss_values = history_dict_2["loss"]
val_loss_values = history_dict_2["val_loss"]
epochs = range(1, len(loss_values) + 1)
plt.plot(epochs, loss_values, "bo", label="Training loss")
plt.plot(epochs, val_loss_values, "ro", label="Validation loss")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Loss")
plt.legend()
plt.title("Training and validation loss")
Text(0.5, 1.0, 'Training and validation loss')
# plot the training and validation accuracy
import matplotlib.pyplot as plt
plt.figure(figsize=(9, 5))
loss_values = history_dict_2["accuracy"]
val_loss_values = history_dict_2["val_accuracy"]
epochs = range(1, len(loss_values) + 1)
plt.plot(epochs, loss_values, "bo", label="Training accuracy")
plt.plot(epochs, val_loss_values, "ro", label="Validation accuracy")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Accuracy")
plt.legend()
plt.title("Training and validation accuracy")
Text(0.5, 1.0, 'Training and validation accuracy')
history_dict_3 = history_3.history
# plot the training and validation loss
import matplotlib.pyplot as plt
plt.figure(figsize=(9, 5))
loss_values = history_dict_3["loss"]
val_loss_values = history_dict_3["val_loss"]
epochs = range(1, len(loss_values) + 1)
plt.plot(epochs, loss_values, "bo", label="Training loss")
plt.plot(epochs, val_loss_values, "ro", label="Validation loss")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Loss")
plt.legend()
plt.title("Training and validation loss")
Text(0.5, 1.0, 'Training and validation loss')
# plot the training and validation accuracy
import matplotlib.pyplot as plt
plt.figure(figsize=(9, 5))
loss_values = history_dict_3["accuracy"]
val_loss_values = history_dict_3["val_accuracy"]
epochs = range(1, len(loss_values) + 1)
plt.plot(epochs, loss_values, "bo", label="Training accuracy")
plt.plot(epochs, val_loss_values, "ro", label="Validation accuracy")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Accuracy")
plt.legend()
plt.title("Training and validation accuracy")
Text(0.5, 1.0, 'Training and validation accuracy')
df = pd.DataFrame({'epochs': epochs, 'model1_val_loss': history_dict_1["val_loss"],
'model2_val_loss': history_dict_2["val_loss"], 'model3_val_loss': history_dict_3["val_loss"],
'model1_val_accuracy': history_dict_1["val_accuracy"], 'model2_val_accuracy': history_dict_2["val_accuracy"],
'model3_val_accuracy': history_dict_3["val_accuracy"]})
df = df.round(3)
df
| epochs | model1_val_loss | model2_val_loss | model3_val_loss | model1_val_accuracy | model2_val_accuracy | model3_val_accuracy | |
|---|---|---|---|---|---|---|---|
| 0 | 1 | 0.427 | 0.417 | 0.471 | 0.849 | 0.850 | 0.822 |
| 1 | 2 | 0.389 | 0.395 | 0.422 | 0.854 | 0.857 | 0.842 |
| 2 | 3 | 0.354 | 0.360 | 0.375 | 0.870 | 0.871 | 0.863 |
| 3 | 4 | 0.350 | 0.358 | 0.349 | 0.870 | 0.879 | 0.874 |
| 4 | 5 | 0.325 | 0.386 | 0.335 | 0.880 | 0.869 | 0.879 |
| 5 | 6 | 0.347 | 0.376 | 0.348 | 0.874 | 0.870 | 0.870 |
| 6 | 7 | 0.333 | 0.381 | 0.324 | 0.870 | 0.878 | 0.885 |
| 7 | 8 | 0.322 | 0.374 | 0.319 | 0.881 | 0.876 | 0.888 |
| 8 | 9 | 0.320 | 0.368 | 0.298 | 0.885 | 0.891 | 0.891 |
| 9 | 10 | 0.328 | 0.398 | 0.333 | 0.883 | 0.884 | 0.884 |
| 10 | 11 | 0.327 | 0.395 | 0.333 | 0.888 | 0.887 | 0.887 |
| 11 | 12 | 0.336 | 0.401 | 0.357 | 0.890 | 0.883 | 0.884 |
| 12 | 13 | 0.317 | 0.385 | 0.353 | 0.889 | 0.890 | 0.888 |
| 13 | 14 | 0.369 | 0.388 | 0.366 | 0.883 | 0.886 | 0.886 |
| 14 | 15 | 0.331 | 0.416 | 0.352 | 0.890 | 0.885 | 0.884 |
| 15 | 16 | 0.323 | 0.439 | 0.409 | 0.891 | 0.888 | 0.889 |
| 16 | 17 | 0.329 | 0.483 | 0.385 | 0.892 | 0.883 | 0.888 |
| 17 | 18 | 0.322 | 0.508 | 0.410 | 0.893 | 0.866 | 0.887 |
| 18 | 19 | 0.348 | 0.471 | 0.425 | 0.892 | 0.886 | 0.889 |
| 19 | 20 | 0.336 | 0.475 | 0.431 | 0.894 | 0.883 | 0.885 |
| 20 | 21 | 0.374 | 0.517 | 0.441 | 0.881 | 0.881 | 0.873 |
| 21 | 22 | 0.383 | 0.480 | 0.432 | 0.880 | 0.886 | 0.893 |
| 22 | 23 | 0.413 | 0.600 | 0.448 | 0.880 | 0.884 | 0.885 |
| 23 | 24 | 0.349 | 0.545 | 0.442 | 0.889 | 0.877 | 0.892 |
| 24 | 25 | 0.412 | 0.532 | 0.463 | 0.886 | 0.880 | 0.892 |
| 25 | 26 | 0.367 | 0.524 | 0.432 | 0.889 | 0.877 | 0.896 |
| 26 | 27 | 0.396 | 0.535 | 0.466 | 0.887 | 0.890 | 0.886 |
| 27 | 28 | 0.392 | 0.586 | 0.453 | 0.890 | 0.888 | 0.891 |
| 28 | 29 | 0.400 | 0.566 | 0.437 | 0.890 | 0.890 | 0.896 |
| 29 | 30 | 0.396 | 0.531 | 0.489 | 0.891 | 0.891 | 0.890 |
print(f"[Model 1] Epoch of minimum validation loss: {np.argmin(history_dict_1['val_loss']) + 1}")
print(f"[Model 2] Epoch of minimum validation loss: {np.argmin(history_dict_2['val_loss']) + 1}")
print(f"[Model 3] Epoch of minimum validation loss: {np.argmin(history_dict_3['val_loss']) + 1}")
[Model 1] Epoch of minimum validation loss: 13 [Model 2] Epoch of minimum validation loss: 4 [Model 3] Epoch of minimum validation loss: 9
| Model | num_layers | batch_size | Best Epoch | min_val_loss | val_accuracy |
|---|---|---|---|---|---|
| Model 1 | 2 (512, 10) | 128 | 13 | 0.317 | 0.889 |
| Model 2 | 2 (512, 10) | 32 | 4 | 0.358 | 0.879 |
| Model 3 | 3 (512, 128, 10) | 128 | 9 | 0.333 | 0.884 |
Model 1: The base model consists of an input layer with 512 neurons and an output layer with 10 neurons. The batch size is 128.
Model 2: This model modifies the base model by changing the batch size to 32 during training.
Model 3: The base model is modified by adding a hidden layer with 128 neurons.
from keras.models import load_model
best_model = load_model('data/best_model_1.hdf5')
y_pred_val = best_model.predict(X_val)
y_pred_val.shape
375/375 [==============================] - 0s 1ms/step
(12000, 10)
y_pred_val_class = y_pred_val.argmax(axis=1)
y_pred_val_class
array([7, 8, 8, ..., 9, 5, 5], dtype=int64)
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
cm = confusion_matrix(y_val, y_pred_val_class)
disp = ConfusionMatrixDisplay(confusion_matrix=cm)
disp.plot()
plt.show()
from sklearn.metrics import classification_report
print(classification_report(y_val, y_pred_val_class))
precision recall f1-score support
0 0.84 0.87 0.85 1232
1 0.99 0.98 0.98 1174
2 0.82 0.83 0.83 1200
3 0.88 0.91 0.90 1242
4 0.82 0.75 0.79 1185
5 0.96 0.97 0.96 1141
6 0.72 0.71 0.71 1243
7 0.97 0.93 0.95 1224
8 0.97 0.98 0.97 1149
9 0.94 0.97 0.96 1210
accuracy 0.89 12000
macro avg 0.89 0.89 0.89 12000
weighted avg 0.89 0.89 0.89 12000
y_pred_test = best_model.predict(X_test)
y_pred_test.shape
313/313 [==============================] - 0s 1ms/step
(10000, 10)
y_pred_test_class = y_pred_test.argmax(axis=1)
y_pred_test_class
array([0, 1, 2, ..., 8, 8, 1], dtype=int64)
cm = confusion_matrix(y_test, y_pred_test_class)
disp = ConfusionMatrixDisplay(confusion_matrix=cm)
disp.plot()
plt.show()
print(classification_report(y_test, y_pred_test_class))
precision recall f1-score support
0 0.82 0.85 0.83 1000
1 0.98 0.99 0.99 1000
2 0.83 0.81 0.82 1000
3 0.92 0.91 0.91 1000
4 0.83 0.81 0.82 1000
5 0.97 0.96 0.96 1000
6 0.70 0.71 0.71 1000
7 0.96 0.92 0.94 1000
8 0.98 0.98 0.98 1000
9 0.93 0.98 0.95 1000
accuracy 0.89 10000
macro avg 0.89 0.89 0.89 10000
weighted avg 0.89 0.89 0.89 10000
from sklearn.metrics import precision_recall_curve
y_scores = y_pred_test
y_scores_5 = y_scores[:, 5]
y_test_5 = (y_test == 5).astype(int)
precisions, recalls, thresholds = precision_recall_curve(y_test_5, y_scores_5)
print(precisions[:5])
print(recalls[:5])
print(thresholds[:5])
[0.1 0.10001 0.10002 0.10003001 0.10004002] [1. 1. 1. 1. 1.] [3.8740736e-20 1.1920793e-19 1.4779576e-19 3.0944573e-19 1.0991552e-18]
# find the threshold for 97% precision
idx = (precisions >= 0.97).argmax()
thresholds[idx]
0.5053034
threshold = 0.50
plt.plot(thresholds, precisions[:-1], "b--", label="Precision", linewidth=2)
plt.plot(thresholds, recalls[:-1], "g-", label="Recall", linewidth=2)
plt.vlines(threshold, 0, 1.0, "k", "dotted", label="threshold")
idx = (thresholds >= threshold).argmax() # first index ≥ threshold
plt.plot(thresholds[idx], precisions[idx], "bo")
plt.plot(thresholds[idx], recalls[idx], "go")
plt.axis([-1, 2, 0, 1])
plt.grid()
plt.xlabel("Threshold")
plt.legend(loc="center right")
plt.show()
from sklearn.metrics import precision_score, recall_score
idx_for_99_precision = (precisions >= 0.99).argmax()
threshold_for_99_precision = thresholds[idx_for_99_precision]
print('threshold for 99% precision: ', threshold_for_99_precision)
y_test_pred_99 = (y_scores_5 >= threshold_for_99_precision)
print('precision score: ', precision_score(y_test_5, y_test_pred_99))
print('recall score: ', recall_score(y_test_5, y_test_pred_99))
threshold for 99% precision: 0.9192923 precision score: 0.9902702702702703 recall score: 0.916
from sklearn.metrics import precision_score, recall_score
idx_for_98_recall = (recalls >= 0.98).argmin() - 1
threshold_for_98_recall = thresholds[idx_for_98_recall]
print('threshold for 98% recall: ', threshold_for_98_recall)
y_test_pred_98 = (y_scores_5 >= threshold_for_98_recall)
print('precision score: ', precision_score(y_test_5, y_test_pred_98))
print('recall score: ', recall_score(y_test_5, y_test_pred_98))
threshold for 98% recall: 0.06795582 precision score: 0.9099350046425255 recall score: 0.98